#!/usr/bin/env python
# encoding=utf-8
import os
import sys
import re

from pymongo import MongoClient
from scpy.logger import get_logger

reload(sys)
sys.setdefaultencoding('utf-8')
logger = get_logger(__file__)
MONGO = MongoClient('10.132.23.104', 27017)
PATTERN = re.compile(ur'[\u4e00-\u9fa5]{10,}')


def pdf2text(path):
    content = os.popen('pdftotext {path} -'.format(path=path))
    return content.read().decode('utf-8')


def build_path(key):
    return '/sc/qiniu/%s/%s' % (key[:4], key)


def main():
    done, removed = 0, 0
    for index, item in enumerate(MONGO['stockProj']['shenhuStockAnno'].find()):
        if ('fileKey' not in item) or (not item['fileKey'].endswith('pdf')):
            continue
        if 'content' in item:
            continue
        logger.info('now index %s, %s' % (index, item['_id']))
        path = build_path(item['fileKey'])
        if not os.path.exists(path):
            continue
        content = pdf2text(build_path(item['fileKey']))

        if PATTERN.search(content) is None:
            MONGO['stockProj']['shenhuStockAnnoParseErr'].insert_one(item)
            continue

        MONGO['stockProj']['shenhuStockAnno'].update_one(
            {'_id': item['_id']},
            {'$set': {'content': content}}
        )
        if not MONGO['qiniu']['todoJobs'].find_one({'key': item['fileKey']}):
            os.remove(path)
            removed += 1
            logger.info('removed %s' % removed)
        done += 1
        logger.info('done %s, %s' % (done, item['_id']))


if __name__ == '__main__':
    # print main()
    print PATTERN.search(u'厕厕厕厕厕厕厕厕厕 厕厕厕厕厕厕厕aregb所arbh')
